package com.wisnews.util.spider;

import com.baomidou.mybatisplus.core.toolkit.ObjectUtils;
import com.baomidou.mybatisplus.core.toolkit.StringUtils;
import com.wisnews.service.sec.INewsService;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;

/**
 * @Description：2021新增网络爬虫
 * @Author: yuyd
 * @Email: yyd8358@foxmail.com
 * @Since: 2021-05-04 9:07
 * @Version
 */
@Component("webMagic2")
@Slf4j
public class WebMagic2 implements PageProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(100).setCharset("utf-8");

    @Autowired
    private INewsService newsService;

    @Override
    public void process(Page page) {

    // 2021/5/4 新增 url规则改变
    // https://www.163.com/dy/article/G91DQIVF0512CCJ3.html
        List<String> newUrls = page.getHtml().links().regex("(https://\\w+\\.163\\.com/\\w+/article/\\w+\\.html)").all();

        page.addTargetRequests(newUrls);
        page.addTargetRequests(newUrls);
        log.info("新版爬虫开始，获取链接总数"+newUrls.size());

        Selectable url = page.getUrl();
        if (StringUtils.isBlank(page.getHtml().$("#container .post_main h1","text").toString())){
            //跳过娱乐新闻，娱乐新闻乱码
            page.setSkip(true);
            log.info("  << 娱乐新闻 >> 或 没有获取到<<新闻标题>> 跳过页面 "+url.toString());
        }

        List<String> urlList = null;
        try {
            urlList = newsService.getAllPageUrl();
        } catch (Exception e) {
            log.error("获取url失败");
        }
        if (ObjectUtils.isNotNull(urlList) && urlList.contains(url.toString())){
            page.setSkip(true);
            log.info("  页面数据库已经存在，跳过页面"+url.toString());
        }

        //存储值  添加过滤规则
        page.putField("newsTitle",page.getHtml().$("#container .post_main h1","text"));
        page.putField("newsAnnounceTime",page.getHtml().$(".post_main .post_info","text"));
        page.putField("newsMedia",page.getHtml().$(".post_main .post_info","text"));
        page.putField("newsContent",page.getHtml().$(".post_main .post_body p").all());
        page.putField("newsType", page.getHtml().$("#container .post_crumb a","text").all());
        page.putField("newsFromUrl",url.toString());

    }


    @Override
    public Site getSite() {
        return site;
    }

}
